package gpl.pierrick.brihaye.aramorph.lucene;
import java.io.IOException;
import java.io.StringReader;
import java.util.HashSet;
import java.util.Set;

import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;

/**
 * A filter that normalizes arabic words to some simpler format.
 * It removes all accents (tashkil) and kashida (tatweel).
 * These rules overcome the most spelling mistakes people do.
 * @author Ahmed Saad
 */
public class Deduplicator extends TokenFilter {
	/**Set of tokens that was found in current location*/
	private Set<String> tokens = new HashSet<String>();

	private PositionIncrementAttribute posAtt;
	private TermAttribute termAtt;
	
	public Deduplicator(TokenStream input) {
		super(input);
		termAtt = (TermAttribute) addAttribute(TermAttribute.class);
		posAtt = (PositionIncrementAttribute) addAttribute(PositionIncrementAttribute.class);
	}

	@Override
	public boolean incrementToken() throws IOException {
		boolean moreTokens;
		while ((moreTokens = input.incrementToken()) &&
				posAtt.getPositionIncrement() == 0 &&
				tokens.contains(termAtt.term())) {
		}
		if (posAtt.getPositionIncrement() > 0)
			tokens.clear();
		tokens.add(termAtt.term());
		return moreTokens;
	}

	public static void main(String[] args) {
//		try {
//			String arabicText = "بسم الله الرحمن الرحيم";
//			ArabicTokenizer at = new ArabicTokenizer(new StringReader(arabicText));
//			Deduplicator dd = new Deduplicator(at);
//			assert dd.incrementToken();
//			assert dd.getAttribute(TermAttribute.class).term().equals("بسم");
//			assert dd.incrementToken();
//			assert dd.getAttribute(TermAttribute.class).term().equals("الله");
//			assert dd.incrementToken();
//			assert dd.getAttribute(TermAttribute.class).term().equals("الرحمن");
//			assert dd.incrementToken();
//			assert dd.getAttribute(TermAttribute.class).term().equals("الرحيم");
//			assert !dd.incrementToken();
//		} catch (IOException e) {
//			e.printStackTrace();
//		}
	}
}
